Example analysis of Python custom process pool [producer and consumer model problems]

This article analyzes the custom Python process pool as an example. I will share it with you for your reference as follows:

Code description 1 cut:

#encoding=utf-8
#author: walker
#date: 2014-05-21
#function:  Custom process pool traverses files under directory 
from multiprocessing import Process, Queue, Lock
import time, os
# consumers 
class Consumer(Process):
  def __init__(self, queue, ioLock):
    super(Consumer, self).__init__()
    self.queue = queue
    self.ioLock = ioLock
  def run(self):
    while True:
      task = self.queue.get()  # When there are no tasks in the queue, the process is blocked 
      if isinstance(task, str) and task == 'quit':
        break;
      time.sleep(1)  # Assume that task handling is required 1 seconds 
      self.ioLock.acquire()
      print( str(os.getpid()) + ' ' + task)
      self.ioLock.release()
    self.ioLock.acquire()
    print 'Bye-bye'
    self.ioLock.release()
# producers 
def Producer():
  queue = Queue()  # This queue is the process / thread-safe 
  ioLock = Lock()
  subNum = 4  # Number of child processes 
  workers = build_worker_pool(queue, ioLock, subNum)
  start_time = time.time()
  for parent, dirnames, filenames in os.walk(r'D:\test'):
    for filename in filenames:
      queue.put(filename)
      ioLock.acquire()
      print('qsize:' + str(queue.qsize()))
      ioLock.release()
      while queue.qsize() > subNum * 10: # Controls the number of tasks in the queue 
        time.sleep(1)
  for worker in workers:
    queue.put('quit')
  for worker in workers:
    worker.join()
  ioLock.acquire()
  print('Done! Time taken: {}'.format(time.time() - start_time))
  ioLock.release()
# Create process pool 
def build_worker_pool(queue, ioLock, size):
  workers = []
  for _ in range(size):
    worker = Consumer(queue, ioLock)
    worker.start()
    workers.append(worker)
  return workers
if __name__ == '__main__':
  Producer()

ps:

self.ioLock.acquire()
...
self.ioLock.release()

Available:

with self.ioLock:
  ...

Alternative.

Here's another fun example:

#encoding=utf-8
#author: walker
#date: 2016-01-06
#function: 1 Interesting examples of multiple processes 
import os, sys, time
from multiprocessing import Pool
cur_dir_fullpath = os.path.dirname(os.path.abspath(__file__))
g_List = ['a']
# Modify global variables g_List
def ModifyDict_1():
  global g_List
  g_List.append('b')
# Modify global variables g_List
def ModifyDict_2():
  global g_List
  g_List.append('c')
# To deal with 1 a 
def ProcOne(num):
  print('ProcOne ' + str(num) + ', g_List:' + repr(g_List))
# Handle all 
def ProcAll():
  pool = Pool(processes = 4)
  for i in range(1, 20):
    #ProcOne(i)
    #pool.apply(ProcOne, (i,))
    pool.apply_async(ProcOne, (i,))
  pool.close()
  pool.join()
ModifyDict_1() # Modify global variables g_List
if __name__ == '__main__':
  ModifyDict_2() # Modify global variables g_List
  print('In main g_List :' + repr(g_List))
  ProcAll()

The result of Windows7:

 Lambda.  python3 demo.py
In main g_List :['a', 'b', 'c']
ProcOne 1, g_List:['a', 'b']
ProcOne 2, g_List:['a', 'b']
ProcOne 3, g_List:['a', 'b']
ProcOne 4, g_List:['a', 'b']
ProcOne 5, g_List:['a', 'b']
ProcOne 6, g_List:['a', 'b']
ProcOne 7, g_List:['a', 'b']
ProcOne 8, g_List:['a', 'b']
ProcOne 9, g_List:['a', 'b']
ProcOne 10, g_List:['a', 'b']
ProcOne 11, g_List:['a', 'b']
ProcOne 12, g_List:['a', 'b']
ProcOne 13, g_List:['a', 'b']
ProcOne 14, g_List:['a', 'b']
ProcOne 15, g_List:['a', 'b']
ProcOne 16, g_List:['a', 'b']
ProcOne 17, g_List:['a', 'b']
ProcOne 18, g_List:['a', 'b']
ProcOne 19, g_List:['a', 'b']

Results of Ubuntu 14.04:

In main g_List :['a', 'b', 'c']
ProcOne 1, g_List:['a', 'b', 'c']
ProcOne 2, g_List:['a', 'b', 'c']
ProcOne 3, g_List:['a', 'b', 'c']
ProcOne 5, g_List:['a', 'b', 'c']
ProcOne 4, g_List:['a', 'b', 'c']
ProcOne 8, g_List:['a', 'b', 'c']
ProcOne 9, g_List:['a', 'b', 'c']
ProcOne 7, g_List:['a', 'b', 'c']
ProcOne 11, g_List:['a', 'b', 'c']
ProcOne 6, g_List:['a', 'b', 'c']
ProcOne 12, g_List:['a', 'b', 'c']
ProcOne 13, g_List:['a', 'b', 'c']
ProcOne 10, g_List:['a', 'b', 'c']
ProcOne 14, g_List:['a', 'b', 'c']
ProcOne 15, g_List:['a', 'b', 'c']
ProcOne 16, g_List:['a', 'b', 'c']
ProcOne 17, g_List:['a', 'b', 'c']
ProcOne 18, g_List:['a', 'b', 'c']
ProcOne 19, g_List:['a', 'b', 'c']

You can see that the second modification of Windows7 was not successful, while the modification of Ubuntu was successful. According to uliweb author limodou, the reason is that Windows is a subprocess that restarts the implementation; Linux is implemented by fork.

More about Python related topics: interested readers to view this site "Python URL skills summary", "Python pictures skills summary", "Python data structure and algorithm tutorial", "Python Socket programming skills summary", "Python function using techniques", "Python string skills summary", "Python introduction and advanced tutorial" and "Python file and directory skills summary"

I hope this article is helpful to you Python programming.